import requests
from bs4 import BeautifulSoup
import pandas as pd
import time

def fetch_page(url):
    """
    使用 requests 库获取网页内容。

    参数:
    url (str): 目标网页的 URL。

    返回:
    str: 网页的 HTML 内容。
    """
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
    }
    response = requests.get(url, headers=headers)
    response.raise_for_status()
    return response.text

def parse_page(html):
    """
    使用 BeautifulSoup 解析网页内容，提取商品信息。

    参数:
    html (str): 网页的 HTML 内容。

    返回:
    list: 包含商品信息的字典列表。
    """
    soup = BeautifulSoup(html, 'html.parser')
    cars = []
    for item in soup.select('.rk-item'):  # Adjust the selector based on actual HTML structure
        name = item.select_one('.rk-car-name').text.strip()
        price = item.select_one('.rk-car-price').text.strip()
        cars.append({'name': name, 'price': price})
    return cars

def clean_data(cars):
    """
    清洗提取的商品信息。

    参数:
    cars (list): 包含商品信息的字典列表。

    返回:
    list: 清洗后的商品信息字典列表。
    """
    for car in cars:
        price = car['price'].replace('万', '').strip()  # Remove '万' and whitespace
        if '-' in price:
            min_price, max_price = map(float, price.split('-'))
            car['price'] = (min_price + max_price) / 2  # Calculate the average price
        else:
            car['price'] = float(price) if price else None
    return cars

def save_to_csv(cars, filename):
    """
    将商品信息保存到 CSV 文件。

    参数:
    cars (list): 包含商品信息的字典列表。
    filename (str): CSV 文件名。
    """
    df = pd.DataFrame(cars)
    df.to_csv(filename, index=False)

if __name__ == '__main__':
    base_url = 'https://car.yiche.com/newcar/salesrank/?page='
    all_cars = []

    for page in range(1, 21):  # Assuming there are at least 20 pages
        url = f"{base_url}{page}"
        print(f"Fetching page {page}")
        page_content = fetch_page(url)
        cars = parse_page(page_content)
        all_cars.extend(cars)
        time.sleep(1)  # Be polite and avoid being blocked

    cleaned_cars = clean_data(all_cars)
    save_to_csv(cleaned_cars, 'cars_price.csv')
    print("Data saved to cars_price.csv")
